package com.incito.crawl.common.utils.http;

import com.incito.crawl.common.httpclient.CrawlHttpClientContext;
import com.incito.crawl.common.httpclient.CrawlerConnectionKeepAliveStrategy;
import com.incito.crawl.common.httpclient.IdleConnectionMonitor;
import com.incito.crawl.common.utils.proxy.ProxyInfo;
import com.incito.crawl.common.utils.proxy.ProxyKit;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.*;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.HttpHostConnectException;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.util.ArrayList;
import java.util.List;

/**
 * Created with IntelliJ IDEA.
 * User: liu
 * Date: 14-5-26
 * Time: 上午11:36
 * To change this template use File | Settings | File Templates.
 */
public class CrawlerKit {

    private static Logger logger = LoggerFactory.getLogger("crawldesc");

    private static PoolingHttpClientConnectionManager connectionManager = createConnectManager();
    private static LaxRedirectStrategy laxRedirectStrategy = new LaxRedirectStrategy();
    private static ConnectionKeepAliveStrategy keepAliveStrategy = new CrawlerConnectionKeepAliveStrategy();
    private static List<BasicHeader> defaultHeaders = new ArrayList<BasicHeader>();
    private static Boolean defaultUseProxy = CrawlHttpClientConfig.getUseProxy();
    private static IdleConnectionMonitor idleConnectionMonitor = new IdleConnectionMonitor("internal", connectionManager, 5000);

    //
//     <util:list id="defaultHeaders">
//    <bean class="org.apache.http.message.BasicHeader">
//     <constructor-arg index="0" value="Accept"/>
//    <constructor-arg index="1" value="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"/>
//    </bean>
//    <bean class="org.apache.http.message.BasicHeader">
//    <constructor-arg index="0" value="Accept-Language"/>
//    <constructor-arg index="1" value="zh-cn,zh,en-US,en;q=0.5"/>
//    </bean>
//    <bean class="org.apache.http.message.BasicHeader">
//    <constructor-arg index="0" value="Accept-Charset"/>
//    <constructor-arg index="1" value="ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7"/>
//    </bean>
//    <bean class="org.apache.http.message.BasicHeader">
//    <constructor-arg index="0" value="Accept-Encoding"/>
//    <constructor-arg index="1" value="gzip, deflate"/>
//    </bean>
//    </util:list>
    static {
        defaultHeaders.add(new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
        defaultHeaders.add(new BasicHeader("Accept-Language", "zh-cn,zh,en-US,en;q=0.5"));
        defaultHeaders.add(new BasicHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7"));
        defaultHeaders.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
    }

    /**
     * 启动内部管理的idleConnectionMonitor
     * 只能启动一次
     */
    public static void connectionMonitorStart() {
        idleConnectionMonitor.start();
    }

    /**
     * 关闭内部管理的idleConnectionMonitor
     */
    public static void connectionMonitorShutdown() {
        idleConnectionMonitor.shutdown();
    }


    public static RequestConfig.Builder createRequestConfigBuilder(Boolean redirectsEnabled, int connectTimeout, int socketTimeout, int maxRedirects) {
        RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
        requestConfigBuilder.setConnectTimeout(connectTimeout);
        requestConfigBuilder.setSocketTimeout(socketTimeout);
        requestConfigBuilder.setRedirectsEnabled(redirectsEnabled);
        requestConfigBuilder.setCircularRedirectsAllowed(true);
        requestConfigBuilder.setRelativeRedirectsAllowed(true);
        requestConfigBuilder.setMaxRedirects(maxRedirects);
        requestConfigBuilder.setCookieSpec("compatibility");
        return requestConfigBuilder;
    }

    public static RequestConfig.Builder createRequestConfigBuilder(Boolean redirectsEnabled) {
        return createRequestConfigBuilder(redirectsEnabled, 3000, 5000, 10);
    }

    /**
     * @param useProxy
     * @return
     */
    public static CloseableHttpClient createHttpClient(Boolean useProxy, Boolean redirectsEnabled) {

        RequestConfig.Builder requestConfigBuilder = createRequestConfigBuilder(redirectsEnabled);
        RequestConfig requestConfig = requestConfigBuilder.build();
        HttpClientBuilder httpClientBuilder = createHttpClientBuilder(requestConfig);
        if (useProxy) {
            ProxyInfo proxyInfo = ProxyKit.getProxy();
            if (proxyInfo != null) {
                logger.info("getProxy-ip:{},port:{}", proxyInfo.getIp(), proxyInfo.getPort());
                HttpHost proxy = new HttpHost(proxyInfo.getIp(), proxyInfo.getPort());
                DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
                httpClientBuilder.setRoutePlanner(routePlanner);
            }
        }

        CloseableHttpClient httpClient = httpClientBuilder.build();
        return httpClient;
    }


    /**
     * <bean id="httpClientBuilder" class="org.apache.http.impl.client.HttpClientBuilder" factory-method="create">
     * <property name="defaultRequestConfig" ref="requestConfig"/>
     * <property name="connectionManager" ref="poolingHttpClientConnectionManager" />
     * <property name="redirectStrategy" ref="laxRedirectStrategy"/>
     * <property name="userAgent" value="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0"/>
     * <property name="defaultHeaders" ref="defaultHeaders" />
     * </bean>
     */

    public static HttpClientBuilder createHttpClientBuilder(RequestConfig requestConfig) {

        HttpClientBuilder clientBuilder = HttpClientBuilder.create();
        clientBuilder.setDefaultRequestConfig(requestConfig);
        clientBuilder.setConnectionManager(connectionManager);
        clientBuilder.setRedirectStrategy(laxRedirectStrategy);
        clientBuilder.setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0");
        clientBuilder.setDefaultHeaders(defaultHeaders);
        clientBuilder.setKeepAliveStrategy(keepAliveStrategy);
        return clientBuilder;

    }

    public static PoolingHttpClientConnectionManager createConnectManager() {
        PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
        connectionManager.setMaxTotal(500);
        connectionManager.setDefaultMaxPerRoute(50);
        return connectionManager;
    }

    public static HttpResp execute(String httpMethod, String url, HttpContext httpContext, BasicHeader[] headers, List<NameValuePair> params, CorrectPage correctPage, Boolean redirectsEnabled, int retry) throws IOException {
        HttpRequestBase httpRequest = null;
        int eStatusCode = 0;
        String msg = null;
        //基于请求方式和url构建httpRequest
        if ("post".equalsIgnoreCase(httpMethod)) {

            httpRequest = new HttpPost(url);
            if (params != null && params.size() > 0) {
                UrlEncodedFormEntity encodedFormEntity = new UrlEncodedFormEntity(params, "utf-8");
                ((HttpPost) httpRequest).setEntity(encodedFormEntity);
            }
        } else {
            if (params != null && params.size() > 0) {
                try {
                    URI uri = new URIBuilder(url).addParameters(params).build();
                    httpRequest = new HttpGet(uri);
                } catch (URISyntaxException e) {
                    logger.error("URL 不符合规范,url={}", url);
                }
            } else {
                httpRequest = new HttpGet(url);
            }
        }

        if (headers != null) {
            for (int i = 0; i < headers.length; i++)
                httpRequest.addHeader(headers[i]);
        }
        HttpResp httpResp = null;
        int retryCount = retry;
        int requestNum = 0;
        int errorNum = 0;
        CloseableHttpResponse response = null;
        while (retryCount > 0) {
            --retryCount;
            requestNum++;
            eStatusCode = 0;
            msg = null;
            CloseableHttpClient httpClient = createHttpClient(defaultUseProxy, redirectsEnabled);
            logger.info("---抓取---" + url + "---第---" + (retry - retryCount) + "---次---");
            //System.out.println("-----分割线-----");
            response = null;
            try {
                response = httpClient.execute(httpRequest, httpContext);
                String html = respToString(response);


                httpResp = new HttpResp(response, html);

                if (response != null) {
                    int statusCode = response.getStatusLine().getStatusCode();
                    if (statusCode == 403 || statusCode == 404) {
                        eStatusCode = statusCode;
                        //网页不可访问，可能是url错误，直接跳出，不再重复尝试
                        break;
                    }
                    HttpEntity entity = response.getEntity();
                    if (entity != null) {
                        Header contentTypeHeader = entity.getContentType();
                        if (contentTypeHeader != null) {
                            String contentType = contentTypeHeader.getValue();
                            httpResp.setContentType(contentType);
                        }
                        httpResp.setContentLength(entity.getContentLength());

                        Header contentEncodingHeader = entity.getContentEncoding();
                        if (contentEncodingHeader != null) {
                            String contentEncoding = contentEncodingHeader.getValue();
                            httpResp.setContentEncoding(contentEncoding);
                        }
                    }
                    httpResp.setStatusCode(statusCode);
                    if (correctPage != null && !correctPage.isCorrectPage(httpResp)) {
                        errorNum++;
                        continue;
                    }
                }

                httpResp.setRequestNum(requestNum);
                httpResp.setErrorNum(errorNum);
                httpResp.setSucNum(1);
                httpResp.setMsg("抓取成功");

                checkReturnSuccessRoute(httpContext, httpResp);

                return httpResp;
            } catch (Exception e) {
                errorNum++;
                if (e instanceof ConnectTimeoutException) {
                    msg = "连接超时异常：" + e.getMessage();
                    logger.error(msg);
                }
                else if (e instanceof HttpHostConnectException) {
                    msg = "连接异常：" + e.getMessage();
                    logger.error(msg);
                }
                else if (e instanceof SocketTimeoutException) {
                    msg = "读取超时异常：" + e.getMessage();
                    logger.error(msg);
                }
                else {
                    msg = "CrawlerKit execute 异常:" + e.getMessage();
                    logger.error(msg);
                }
                eStatusCode = 500;
                httpRequest.removeHeaders("X-Forwarded-For");
                if (httpContext != null) {
                    httpContext.removeAttribute(HttpClientContext.HTTP_ROUTE);
                    httpContext.removeAttribute(CrawlHttpClientConfig.PROXY_USE_COUNT);
                }

            } finally {
                if (httpResp == null || httpResp.getHtml() == null || StringUtils.isBlank(httpResp.getHtml()) ||
                        (httpResp.getHttpResponse() != null && httpResp.getHttpResponse().getStatusLine() != null
                                && httpResp.getHttpResponse().getStatusLine().getStatusCode() >= 400)
                        || (correctPage != null && !correctPage.isCorrectPage(httpResp))) {
                    httpRequest.removeHeaders("X-Forwarded-For");
                    if (httpContext != null) {
                        httpContext.removeAttribute(HttpClientContext.HTTP_ROUTE);
                        httpContext.removeAttribute(CrawlHttpClientConfig.PROXY_USE_COUNT);
                    }
                }

                try {
                    response.close();
                } catch (Exception e) {
                    //no-op
                }

            }

        }
        HttpResp httpResp1 = new HttpResp(null, null);
        httpResp1.setStatusCode(eStatusCode);
        httpResp1.setMsg(msg);
        return httpResp1;

    }

    /**
     * @param httpClient
     * @param httpMethod         http请求方式名称，get,post
     * @param url                抓取的url
     * @param httpContext        http访问的上下文，httpclient会自动基于它维护reffer及cookie等信息
     * @param headers            访问用到的headers
     * @param correctPage        用于是否正常页的判断, 可以为空，为空不做判断
     * @param retry              当抓取失败或被correctPage.isCorrectPage判定为false时，则重试，retry表示最大重试次数。
     *                           一般地，HttpClient会自动判断网络异常等造成的访问失败，并基于其框架内的RetryHandler来自动重试，
     *                           这里的重试次数更多描述的是当服务端因为某种状态的异常导致的失败或没有返回所需要的页面时的失败下的重试。
     *                           一般不宜超过5次。
     * @param defaultCharsetName 网页正文的字符编码
     * @return
     * @throws IOException
     */
    public static HttpResp execute(CloseableHttpClient httpClient, String httpMethod, String url, HttpContext httpContext, BasicHeader[] headers, List<NameValuePair> params, CorrectPage correctPage, int retry, String defaultCharsetName) throws IOException {
        HttpRequestBase httpRequest = null;
        int eStatusCode = 0;
        String msg = null;
        //基于请求方式和url构建httpRequest
        if ("post".equalsIgnoreCase(httpMethod)) {
            httpRequest = new HttpPost(url);
            Charset charset = null;
            try {
                charset = Charset.forName(defaultCharsetName);
            } catch (UnsupportedCharsetException e) {
                logger.warn(e.getMessage());
                charset = Charset.forName("utf-8");
            }
            if (params != null && params.size() > 0) {
                UrlEncodedFormEntity encodedFormEntity = new UrlEncodedFormEntity(params, charset);
                ((HttpPost) httpRequest).setEntity(encodedFormEntity);
            }
        } else {
            if (params != null && params.size() > 0) {
                try {
                    URI uri = new URIBuilder(url).addParameters(params).build();
                    httpRequest = new HttpGet(uri);
                } catch (URISyntaxException e) {
                    logger.error("URL 不符合规范,url={}", url);
                }
            } else {
                httpRequest = new HttpGet(url);
            }
        }

        if (headers != null) {
            for (int i = 0; i < headers.length; i++)
                httpRequest.addHeader(headers[i]);
        }
        HttpResp httpResp = null;
        int retryCount = retry;
        int requestNum = 0;
        int errorNum = 0;
        CloseableHttpResponse response = null;
        while (retryCount > 0) {
            --retryCount;
            requestNum++;
            eStatusCode = 0;
            msg = null;
            logger.info("---抓取---" + url + "---第---" + (retry - retryCount) + "---次---");
            response = null;
            try {
                response = httpClient.execute(httpRequest, httpContext);

                String html = respToString(response, defaultCharsetName);
                httpResp = new HttpResp(response, html);


                if (response != null) {
                    int statusCode = response.getStatusLine().getStatusCode();
                    if (statusCode == 403 || statusCode == 404) {
                        eStatusCode = statusCode;
                        //网页不可访问，可能是url错误，直接跳出，不再重复尝试
                        break;
                    }
                    HttpEntity entity = response.getEntity();
                    if (entity != null) {
                        Header contentTypeHeader = entity.getContentType();
                        if (contentTypeHeader != null) {
                            String contentType = contentTypeHeader.getValue();
                            httpResp.setContentType(contentType);
                        }
                        httpResp.setContentLength(entity.getContentLength());

                        Header contentEncodingHeader = entity.getContentEncoding();
                        if (contentEncodingHeader != null) {
                            String contentEncoding = contentEncodingHeader.getValue();
                            httpResp.setContentEncoding(contentEncoding);
                        }
                    }
                    httpResp.setStatusCode(statusCode);
                    if (correctPage != null && !correctPage.isCorrectPage(httpResp)) {
                        errorNum++;
                        continue;
                    }
                }


                httpResp.setRequestNum(requestNum);
                httpResp.setErrorNum(errorNum);
                httpResp.setSucNum(1);
                checkReturnSuccessRoute(httpContext, httpResp);

                return httpResp;
            } catch (Exception e) {
                errorNum++;
                if (e instanceof ConnectTimeoutException) {
                    msg = "连接超时异常：" + e.getMessage();
                    logger.error(msg);
                }
                else if (e instanceof HttpHostConnectException) {
                    msg = "连接异常：" + e.getMessage();
                    logger.error(msg);
                }
                else if (e instanceof SocketTimeoutException) {
                    msg = "读取超时异常：" + e.getMessage();
                    logger.error(msg);
                }
                else {
                    msg = "CrawlerKit execute 异常:" + e.getMessage();
                    logger.error(msg,e);
                }
                eStatusCode=500;
                httpRequest.removeHeaders("X-Forwarded-For");
                if (httpContext != null) {
                    httpContext.removeAttribute(HttpClientContext.HTTP_ROUTE);
                    httpContext.removeAttribute(CrawlHttpClientConfig.PROXY_USE_COUNT);
                }
            } finally {

                if (httpResp == null || httpResp.getHtml() == null || StringUtils.isBlank(httpResp.getHtml()) ||
                        (httpResp.getHttpResponse() != null && httpResp.getHttpResponse().getStatusLine() != null
                                && httpResp.getHttpResponse().getStatusLine().getStatusCode() >400)
                        || (correctPage != null && !correctPage.isCorrectPage(httpResp))) {
                    httpRequest.removeHeaders("X-Forwarded-For");
                    if (httpContext != null) {
                        httpContext.removeAttribute(HttpClientContext.HTTP_ROUTE);
                        httpContext.removeAttribute(CrawlHttpClientConfig.PROXY_USE_COUNT);
                    }
                }
                try {
                    response.close();
                } catch (Exception e) {
                    //no-op
                }
            }
        }
        HttpResp httpResp1 = new HttpResp(null, null,null, requestNum, errorNum);
        httpResp1.setStatusCode(eStatusCode);
        httpResp1.setMsg(msg);
        return httpResp1;
    }

    public static HttpResp execute(CloseableHttpClient httpClient, String url, HttpContext httpContext, BasicHeader[] headers, List<NameValuePair> params, CorrectPage correctPage, int retry, String defaultCharset) throws IOException {
        return execute(httpClient, "get", url, httpContext, headers, params, correctPage, retry, defaultCharset);
    }

    public static HttpResp execute(CloseableHttpClient httpClient, String url, HttpContext httpContext, BasicHeader[] headers, CorrectPage correctPage, int retry, String defaultCharset) throws IOException {
        return execute(httpClient, "get", url, httpContext, headers, null, correctPage, retry, defaultCharset);
    }


    /**
     * @param httpClient
     * @param url         抓取的url
     * @param httpContext http访问的上下文，httpclient会自动基于它维护reffer及cookie等信息
     * @param headers     访问用到的headers
     * @param correctPage 用于是否正常页的判断, 可以为空，为空不做判断
     * @param retry       当抓取失败或被correctPage.isCorrectPage判定为false时，则重试，retry表示最大重试次数。
     *                    一般地，HttpClient会自动判断网络异常等造成的访问失败，并基于其框架内的RetryHandler来自动重试，
     *                    这里的重试次数更多描述的是当服务端因为某种状态的异常导致的失败或没有返回所需要的页面时的失败下的重试。
     *                    一般不宜超过5次。
     * @return
     * @throws IOException
     */
    public static HttpResp execute(CloseableHttpClient httpClient, String url, HttpContext httpContext, BasicHeader[] headers, CorrectPage correctPage, int retry) throws IOException {
        return execute(httpClient, url, httpContext, headers, correctPage, retry, "utf-8");
    }

    /**
     * @param httpClient
     * @param url         抓取url
     * @param httpContext http访问的上下文，httpclient会自动基于它维护reffer及cookie等信息
     * @param headers     访问用到的headers
     * @return
     * @throws IOException
     */
    public static HttpResp execute(CloseableHttpClient httpClient, String url, HttpContext httpContext, BasicHeader[] headers) throws IOException {
        return execute(httpClient, url, httpContext, headers, null, 3, "utf-8");
    }

    public static HttpResp execute(String url, HttpContext httpContext, BasicHeader[] headers, List<NameValuePair> params, CorrectPage correctPage, Boolean redirectsEnabled, int retry) throws IOException {
        return execute("get", url, httpContext, headers, params, correctPage, redirectsEnabled, retry);
    }

    public static HttpResp execute(String url, HttpContext httpContext, BasicHeader[] headers, CorrectPage correctPage, Boolean redirectsEnabled, int retry) throws IOException {
        return execute("get", url, httpContext, headers, null, correctPage, redirectsEnabled, retry);
    }

    public static HttpResp execute(String url, HttpContext httpContext, BasicHeader[] headers) throws IOException {
        return execute(url, httpContext, headers, null, false, 3);
    }

    /**
     * 抓取图片
     * @param httpClient
     * @param url
     * @param httpContext
     * @param headers
     * @param params
     * @param connectTimeout
     * @param socketTimeout
     * @param retry
     * @return
     * @throws IOException
     */
    public static HttpResp getImage(CloseableHttpClient httpClient, String url, HttpContext httpContext, BasicHeader[] headers, List<NameValuePair> params,int connectTimeout,int socketTimeout, int retry) throws IOException {
        CloseableHttpResponse response = null;
        HttpRequestBase httpRequest = null;
        int eStatusCode = 0;
        String msg = null;
        if (params != null && params.size() > 0) {
            try {
                URI uri = new URIBuilder(url).addParameters(params).build();
                httpRequest = new HttpGet(uri);
            } catch (URISyntaxException e) {
                logger.error("URL 不符合规范,url={}", url);
            }
        } else {
            httpRequest = new HttpGet(url);
        }

        if (connectTimeout <=0) connectTimeout=3000;
        if (socketTimeout<=0) socketTimeout=15000;

        RequestConfig requestConfig = CrawlerKit.createRequestConfigBuilder(false, connectTimeout, socketTimeout, 10).build();
        httpRequest.setConfig(requestConfig);

        if (headers != null) {
            for (int i = 0; i < headers.length; i++)
                httpRequest.addHeader(headers[i]);
        }
        int retryCount = retry;
        int requestNum = 0;
        int errorNum = 0;
        while (retryCount > 0) {
            --retryCount;
            requestNum++;
            eStatusCode = 0;
            msg = null;
            logger.info("---抓取图片---" + url + "---第---" + (retry - retryCount) + "---次---");
            response = null;
            try {
                response = httpClient.execute(httpRequest, httpContext);

                if (response != null && response.getEntity() !=null) {
                    int statusCode = response.getStatusLine().getStatusCode();
                    if (statusCode == 403 || statusCode == 404 ) {
                        eStatusCode = statusCode;
                        //网页不可访问，可能是url错误，直接跳出，不再重复尝试
                        break;
                    }
                    if (statusCode != 200) {
                        continue;
                    }

                    Header contentTypeHeader = response.getEntity().getContentType();
                    String contentType = contentTypeHeader.getValue();
                    if (contentType != null ) {
                        contentType = contentType.toLowerCase();
                        if (contentType.contains("image")) {

                            HttpEntity entity = response.getEntity();
                            byte[] imageBytes = EntityUtils.toByteArray(entity);

                            HttpResp httpResp = new HttpResp(response,null,imageBytes,requestNum,errorNum);
                            httpResp.setSucNum(1);

                            httpResp.setContentType(contentType);
                            httpResp.setContentLength(entity.getContentLength());

                            Header contentEncodingHeader = entity.getContentEncoding();
                            if (contentEncodingHeader != null) {
                                String contentEncoding = contentEncodingHeader.getValue();
                                httpResp.setContentEncoding(contentEncoding);
                            }
                            httpResp.setStatusCode(statusCode);

                            return httpResp;
                        }
                    }
                }
            } catch (Exception e) {
                errorNum++;
                if (e instanceof ConnectTimeoutException) {
                    msg = "连接超时异常：" + e.getMessage();
                    logger.error(msg);
                }
                else if (e instanceof HttpHostConnectException) {
                    msg = "连接异常：" + e.getMessage();
                    logger.error(msg);
                }
                else if (e instanceof SocketTimeoutException) {
                    msg = "读取超时异常：" + e.getMessage();
                    logger.error(msg);
                }
                else {
                    msg = "CrawlerKit execute 异常:" + e.getMessage();
                    logger.error(msg);
                }
                eStatusCode = 500;
                httpRequest.removeHeaders("X-Forwarded-For");
                if (httpContext != null) {
                    httpContext.removeAttribute(HttpClientContext.HTTP_ROUTE);
                    httpContext.removeAttribute(CrawlHttpClientConfig.PROXY_USE_COUNT);
                }
            } finally {
                if (response == null ||
                        (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() >= 400)) {
                    httpRequest.removeHeaders("X-Forwarded-For");
                    if (httpContext != null) {
                        httpContext.removeAttribute(HttpClientContext.HTTP_ROUTE);
                        httpContext.removeAttribute(CrawlHttpClientConfig.PROXY_USE_COUNT);
                    }
                }

                try {
                    response.close();
                } catch (Exception e) {
                    //no-op
                }


            }
        }
        HttpResp httpResp1 = new HttpResp(null, null,null, requestNum, errorNum);
        httpResp1.setStatusCode(eStatusCode);
        httpResp1.setMsg(msg);
        return httpResp1;
    }

    /**
     * 默认连接超时(3s)和读取超时(15s)的抓取图片方法
     * @param httpClient
     * @param url
     * @param httpContext
     * @param headers
     * @param params
     * @param retry
     * @return
     * @throws IOException
     */
    public static HttpResp getImage(CloseableHttpClient httpClient, String url, HttpContext httpContext, BasicHeader[] headers, List<NameValuePair> params, int retry) throws IOException {
        return getImage(httpClient,url,httpContext,headers,params,3000,15000,retry);
    }


        public static String respToString(CloseableHttpResponse httpResponse, String defaultCharset) throws IOException {
        try {
            String html = EntityUtils.toString(httpResponse.getEntity(), defaultCharset);
            return html;
        } catch (Exception e) {
            return null;
        } finally {
            httpResponse.close();
        }
    }

    public static String respToString(CloseableHttpResponse httpResponse) throws IOException {
        return respToString(httpResponse, "utf-8");
    }

    //检查并返回成功且效果好的代理到代理池中
    private static void checkReturnSuccessRoute(HttpContext httpContext, HttpResp httpResp) {
        CloseableHttpResponse response = httpResp.getHttpResponse();
        String html = httpResp.getHtml();
        if (httpContext != null && response != null && response.getStatusLine().getStatusCode() == 200 && StringUtils.isNotBlank(html)) {
            returnProxyInfoToProxyQueue(httpContext);
        }
    }

    //返回代理到代理池中
    // Note：通过调用ProxyKit里边的同名方法returnProxyInfoToProxyQueue
    private static void returnProxyInfoToProxyQueue(HttpContext httpContext) {
        if (httpContext != null) {
            try {
                HttpRoute httpRoute = (HttpRoute) ((HttpClientContext) httpContext).getHttpRoute();
                if (httpRoute == null) return;
                HttpHost httpHost = httpRoute.getProxyHost();
                if (httpHost == null) return;
                ProxyInfo proxyInfo = new ProxyInfo();
                proxyInfo.setIp(httpHost.getHostName());
                proxyInfo.setPort(httpHost.getPort());
                String proxyRuleName = null;
                if (httpContext instanceof CrawlHttpClientContext) {
                    CrawlHttpClientContext crawlHttpClientContext = (CrawlHttpClientContext) httpContext;
                    proxyRuleName = crawlHttpClientContext.getProxyRuleName();
                }

                ProxyKit.returnProxyInfoToProxyQueue(proxyInfo, proxyRuleName);
            } catch (Exception e) {
                logger.error("获取httpContext中的代理异常：" + e.getMessage(), e);
            }
        }
    }


}
